#  -*- coding: utf-8 -*-

from datetime import datetime
import time,json
import pandas as pd
import os,ast

from pymongo import MongoClient, ASCENDING, DESCENDING, UpdateOne
from HunterHouse.database import DB_CONN

"""
从爬虫生成的json文件中解析数据，保存到本地的MongoDB数据库中
"""

class ProcessData:
    def __init__(self):
        self.sell_house = DB_CONN['sell_house']
        self.rent_house = DB_CONN['rent_house']
        self.community = DB_CONN['community']

    @staticmethod
    def process_community(single_data):
        #print(single_data)
        index = dict()
        doc = dict()
        doc['url'] = single_data['url']
        if len(single_data['name']) > 0:
            doc['name'] = single_data['name'][0]
        else:
            doc['name'] ="无"


        if len(single_data['address']) > 0:
            doc['address'] = single_data['address'][0]

        if len(single_data['location']) > 3:
            if "小区" in single_data['location'][2]:
                doc['section'] = single_data['location'][2].strip("小区")
            if "小区" in single_data['location'][3]:
                doc['street'] = single_data['location'][3].strip("小区")

        if len(single_data['avg_price']) > 1:
            if "月参考均价" in single_data['avg_price'][1]:
                doc['month'] = single_data['avg_price'][1].strip("月参考均价")

            doc['price'] = float(single_data['avg_price'][0])

        if len(single_data['community_info']) > 6:
            #print(single_data['community_info'][0])
            if "年建成" in single_data['community_info'][0]:
                doc['building_time'] = single_data['community_info'][0].strip("年建成 ")
                #print(doc['building_time'])

            doc['building_type'] = single_data['community_info'][1]
            doc['manage_price'] = single_data['community_info'][2]
            doc['manage_company'] = single_data['community_info'][3]
            doc['building_company'] = single_data['community_info'][4]
            if "栋" in single_data['community_info'][5]:
                doc['building_num'] = int(single_data['community_info'][5].strip("栋"))
            if "户" in single_data['community_info'][6]:
                doc['house_num'] = int(single_data['community_info'][6].strip("户"))

        if len(single_data['follow_num']) > 0:
            doc['follow_num'] = int(single_data['follow_num'][0])

        if 'month' not in doc.keys():
            doc['month'] = '0'
        index['url'] = doc['url']
        index['month'] = doc['month']
        index['name'] = doc['name']

        #print(index)
        return index,doc

    @staticmethod
    def process_rent_house(single_data):
        index = dict()
        doc = dict()
        doc['url'] = single_data['url']
        if len(single_data['title']) > 0:
            doc['title'] = single_data['title'][0]
        else:
            doc['title'] = "无"

        if len(single_data['publish_time']) > 1:
            if "房源上架时间" in single_data['publish_time'][1]:
                time = single_data['publish_time'][1].strip()
                time = single_data['publish_time'][1].lstrip("房源上架时间")
                doc['publish_time'] = time

        doc['houseid'] = "无"
        if len(single_data['houseid']) > 0:
            if "房源编号：" in single_data['houseid'][0]:
                doc['houseid'] = single_data['houseid'][0].lstrip("房源编号：")

        doc['price'] = "无"
        if len(single_data['price']) > 0:
            doc['price'] = float(single_data['price'][0])

        if len(single_data['house_basic_info']) > 3:
            doc['rent_type'] = single_data['house_basic_info'][0]
            doc['house_structure'] = single_data['house_basic_info'][1]
            doc['house_area'] = float(single_data['house_basic_info'][2].strip("㎡"))
            doc['house_direction'] = single_data['house_basic_info'][3]


        doc['house_detail_info'] = single_data['house_detail_info']

        index['houseid'] = doc['houseid']
        index['price'] = doc['price']


        #print(index)
        return index, doc

    @staticmethod
    def process_sell_house(single_data):
        index = dict()
        doc = dict()
        doc['url'] = single_data['url']

        if len(single_data['followcnt']) > 0:
            doc['followcnt'] = int(single_data['followcnt'][0])
        else:
            doc['followcnt'] = 0

        if len(single_data['cartcnt']) > 0:
            doc['cartcnt'] = int(single_data['cartcnt'][0])
        else:
            doc['cartcnt'] = 0

        if len(single_data['totalprice_num']) > 0:
            doc['totalprice'] = float(single_data['totalprice_num'][0])
        else:
            doc['totalprice'] = "无"

        if len(single_data['unitprice_num']) > 0:
            doc['unitprice'] = float(single_data['unitprice_num'][0])
        else:
            doc['unitprice'] = 0

        if len(single_data['houseinfo_areainfo']) > 1:
            doc['house_section'] = single_data['houseinfo_areainfo'][0]
            doc['house_street'] = single_data['houseinfo_areainfo'][1]
        else:
            doc['house_section'] = "无"
            doc['house_street'] = "无"

        if len(single_data['communityname']) > 0:
            doc['communityname'] = single_data['communityname'][0]
        else:
            doc['communityname'] = "无"

        if len(single_data['houseID']) > 0:
            doc['houseid'] = single_data['houseID'][0]
        else:
            doc['houseid'] = "无"

        if len(single_data['baseinfo']) > 11:
            doc['structure'] = single_data['baseinfo'][0]
            doc['layer'] = single_data['baseinfo'][1]
            doc['area'] = float(single_data['baseinfo'][2].strip("㎡"))
            doc['layer_type'] = single_data['baseinfo'][3]
            doc['building_type'] = single_data['baseinfo'][5]
            doc['direction'] = single_data['baseinfo'][6]
            doc['building_material'] = single_data['baseinfo'][7]
            doc['building_district'] = single_data['baseinfo'][9]
            doc['building_years'] = single_data['baseinfo'][11]

        if len(single_data['tradeinfo']) > 15:
            doc['publish_time'] = single_data['tradeinfo'][1]
            doc['trade_type'] = single_data['tradeinfo'][3]
            doc['using_type'] = single_data['tradeinfo'][7]

        doc['houseinfo_feature'] = single_data['houseinfo_feature']

        index['houseid'] = doc['houseid']
        index['totalprice'] = doc['totalprice']

        # print(index)
        return index, doc

    def process_single_data(self,jsonfile,single_data):
        result = True
        if "南京小区具体信息" in jsonfile:
            collection = self.community
            index, doc = self.process_community(single_data)
            if index['name'] == "无":
                result = False

        elif "南京租房房源具体信息"in jsonfile:
            collection = self.rent_house
            index, doc = self.process_rent_house(single_data)
            if index['houseid'] == "无" or index['price'] == "无":
                result = False

        elif "南京二手房房源具体信息" in jsonfile:
            collection = self.sell_house
            index, doc = self.process_sell_house(single_data)
            if index['houseid'] == "无" or index['totalprice'] == "无":
                result = False
        else:
            result = False

        return result,index,doc,collection

    def process_json(self, jsonfile):

        with open(jsonfile, 'r', encoding='UTF-8') as f:
            lines = f.readlines()
        update_requests = []
        # json解出后以字典形式存放
        for line in lines:
            single_data = ast.literal_eval(line)
            result,index,doc,collection = self.process_single_data(jsonfile,single_data)
            if result is False:
                continue
            update_requests.append(
                UpdateOne(
                    index,
                    {'$set': doc},
                    upsert=True)
            )
        if  collection == self.community:
            collection.create_index([('name', 1), ('month', 1)])
            collection.create_index([('name', 1),('url', 1), ('month', 1)])

        if  collection == self.rent_house:
            collection.create_index([('houseid', 1), ('price', 1)])

        if  collection == self.sell_house:
            collection.create_index([('houseid', 1), ('totalprice', 1)])

        if len(update_requests) > 0:
            start_time = time.time()
            update_result = collection.bulk_write(update_requests, ordered=False)
            end_time = time.time()
            print('保存数据到数据集：%s 插入：%4d条, 更新：%4d条,耗时：%.3f 秒' %
                  (collection.name, update_result.upserted_count, update_result.modified_count,
                   (end_time - start_time)),
                  flush=True)


if __name__ == '__main__':
    process_data = ProcessData()
    process_data.process_json(u"../南京小区具体信息.json")
    #process_data.process_json(u"../南京租房房源具体信息.json")
    #process_data.process_json(u"../南京二手房房源具体信息.json")

    # process_data.community.create_index([('name', 1), ('month', 1)])
    # process_data.community.create_index([('name', 1), ('url', 1), ('month', 1)])
    # process_data.rent_house.create_index([('houseid', 1), ('price', 1)])
